package crawler.cnki.task;

import crawler.cnki.selenium.SeleniumDownloader;
import crawler.cnki.selenium.WebDriverPool;
import crawler.cnki.utils.AddPaper;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.RedisScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.io.UnsupportedEncodingException;
import java.util.List;

@Component
public class PaperProcessor implements PageProcessor {
    @Autowired
    private DataPipeline dataPipeline;
    @Autowired
    private JedisPool jedisPool;
    @Autowired
    private Spider spider;

    private String root = "http://s.wanfangdata.com.cn/periodical?q";
    private String url1 = "http://s.wanfangdata.com.cn/periodical?q=%E6%95%B0%E6%8D%AE%E5%BA%93";
    private String url2 = "http://s.wanfangdata.com.cn/periodical?q=%E5%9F%BA%E4%BA%8E";

    @Scheduled(fixedDelay = 1000 * 1000)
    public void PaperStarter() {
        new Thread(){
            @Override
            public void run() {
                try {
                    Thread.sleep(60*1000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                spider.run();
            }
        }.start();
        Spider.create(new PaperProcessor())
                .setUUID("Paper")
                .setScheduler(new RedisScheduler(jedisPool))
//                .addUrl(AddPaper.urlMaker());
                .setDownloader(new SeleniumDownloader(0,PoolUtil.getWebDriverPool(), new DataAction()))
//                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .addPipeline(dataPipeline)
                .thread(3)
                .run();
    }

    @Override
    public void process(Page page) {
        Html html = page.getHtml();
        String currentUrl = page.getUrl().toString();
        List<String> links;

        if (currentUrl.contains("d.wanfangdata") && currentUrl.contains("periodical/")) {
            page.putField("itemType", 1);
            Document detailInfo = Jsoup.parse(html.css("div.detailInfo").toString());
            //获取标题
            String title = detailInfo.select("div.detailTitle > div.detailTitelInner > span.detailTitleCN").text();
            page.putField("title", title);

            //作者
            String author = detailInfo.select("div.author div.itemUrl").text().replaceAll(" ", ",");
            page.putField("author", author);

            //摘要
            Elements summarys = detailInfo.select("div.summary");
            if (!summarys.isEmpty()) {
                String summary = summarys.first().ownText();
                page.putField("summary", summary);
            }
            //url
            page.putField("url", currentUrl);

            //doi
            Elements elements = detailInfo.select("div.doi div.itemUrl");
            String doi;
            if (elements.size() > 0) {
                doi = elements.first().text();
                page.putField("DOI", doi);
            }

            //关键词
            Elements keys = detailInfo.select("div.keyword div.itemUrl");
            if (keys.size() == 0)
                keys = detailInfo.select("div.keywordEN div.itemUrl");
            String keywords = keys.text().replaceAll(" ", ";");
//            String[] split = keywords.split(";");
//            if(split.length>=1) {
//                try {
//                    page.addTargetRequest(root+java.net.URLEncoder.encode(split[0],"utf-8"));
//                } catch (UnsupportedEncodingException e) {
//                    e.printStackTrace();
//                }
//            }
            page.putField("keywords", keywords);

            //出版日期
            Elements dates = detailInfo.select("div.publish div.itemUrl");
            if (!dates.isEmpty()) {
                String date = dates.first().ownText();
                page.putField("date", date);
            }
            //所属期刊
            String journal = detailInfo.select("div.serialTitle div.itemUrl").text();
            page.putField("journal", journal);

            //机构
            Elements organizations = detailInfo.select("div.organization div.itemUrl span");
            StringBuilder organization = new StringBuilder();
            for (int i = 0; i < organizations.size(); i = i + 2) {
                organization.append(organizations.get(i).text() + ",");
            }
            if (organization.length() > 0)
                page.putField("organization", organization.substring(0, organization.length() - 1));


//                List<String> list = html.css("div.detailList div.keyword").links().all();
//                page.addTargetRequests(list);
//                list = html.css("div.detailList div.keywordEN").links().all();
//                page.addTargetRequests(list);
            List<String> list = html.css("div.author div.itemUrl").links().all();
            ResearcherProcessor.addurl(list);
            list = html.css("div.contentPaper div.contentInfo > a.title").links().all();
            page.addTargetRequests(list);
//                List<String> OrganizationList = html.css("div.thesisOrganization").links().all();
//                page.addTargetRequests(OrganizationList);

        }else{//搜索页面
            links= html.css("div.result-list-container div.title-area").links().all();
            if(links.size()==0) return;
            page.addTargetRequests(links);
            if(!currentUrl.contains("&p=")){
                page.addTargetRequest(currentUrl+"&p=2");
            }else{
                String s = currentUrl;
                int i = Integer.parseInt(s.substring(s.lastIndexOf("=")+1));
                s = s.substring(0,s.lastIndexOf("=")+1)+(i+1);
                page.addTargetRequest(s);
            }
        }
    }

    private Site site = Site.me()
            .setCharset("utf8")
            .setTimeOut(10 * 1000)
            .setRetrySleepTime(3000)
            .setRetryTimes(3);

    @Override
    public Site getSite() {
        return site;
    }
}
